{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/optimize.py:187: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/models/research/neural_stack.py:52: The name tf.nn.rnn_cell.RNNCell is deprecated. Please use tf.compat.v1.nn.rnn_cell.RNNCell instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/trainer_lib.py:111: The name tf.OptimizerOptions is deprecated. Please use tf.compat.v1.OptimizerOptions instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/trainer_lib.py:111: The name tf.OptimizerOptions is deprecated. Please use tf.compat.v1.OptimizerOptions instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_gan/python/estimator/tpu_gan_estimator.py:42: The name tf.estimator.tpu.TPUEstimator is deprecated. Please use tf.compat.v1.estimator.tpu.TPUEstimator instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_gan/python/estimator/tpu_gan_estimator.py:42: The name tf.estimator.tpu.TPUEstimator is deprecated. Please use tf.compat.v1.estimator.tpu.TPUEstimator instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tensor2tensor import models\n",
    "from tensor2tensor import problems\n",
    "from tensor2tensor.layers import common_layers\n",
    "from tensor2tensor.utils import trainer_lib\n",
    "from tensor2tensor.utils import t2t_model\n",
    "from tensor2tensor.utils import registry\n",
    "from tensor2tensor.utils import metrics\n",
    "from tensor2tensor.data_generators import problem\n",
    "from tensor2tensor.data_generators import text_problems\n",
    "from tensor2tensor.data_generators import translate\n",
    "from tensor2tensor.utils import registry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "@registry.register_problem\n",
    "class TRANSLATION32k(translate.TranslateProblem):\n",
    "\n",
    "    @property\n",
    "    def additional_training_datasets(self):\n",
    "        \"\"\"Allow subclasses to add training datasets.\"\"\"\n",
    "        return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "PROBLEM = 'translatio_n32k'\n",
    "problem = problems.problem(PROBLEM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('t2t/data/vocab.translatio_n32k.32768.subwords',\n",
       " 't2t/train-large/model.ckpt-225000')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "vocab_file = \"t2t/data/vocab.translatio_n32k.32768.subwords\"\n",
    "ckpt_path = tf.train.latest_checkpoint(os.path.join('t2t/train-large'))\n",
    "vocab_file, ckpt_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from t import text_encoder\n",
    "\n",
    "encoder = text_encoder.SubwordTextEncoder(vocab_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py:2262: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py:2262: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting T2TModel mode to 'infer'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting T2TModel mode to 'infer'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.label_smoothing to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.label_smoothing to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.layer_prepostprocess_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.layer_prepostprocess_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.symbol_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.symbol_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.attention_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.attention_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.relu_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.relu_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/layers/common_layers.py:100: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/layers/common_layers.py:100: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py:245: The name tf.summary.text is deprecated. Please use tf.compat.v1.summary.text instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py:245: The name tf.summary.text is deprecated. Please use tf.compat.v1.summary.text instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.get_variable_scope is deprecated. Please use tf.compat.v1.get_variable_scope instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.get_variable_scope is deprecated. Please use tf.compat.v1.get_variable_scope instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.summary.scalar is deprecated. Please use tf.compat.v1.summary.scalar instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.summary.scalar is deprecated. Please use tf.compat.v1.summary.scalar instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py:2248: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py:2248: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py:1373: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py:1373: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.logging.warning is deprecated. Please use tf.compat.v1.logging.warning instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.logging.warning is deprecated. Please use tf.compat.v1.logging.warning instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_26088_1024.bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_26088_1024.bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_26088_1024.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_26088_1024.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.mod is deprecated. Please use tf.math.mod instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.mod is deprecated. Please use tf.math.mod instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/models/transformer.py:95: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/models/transformer.py:95: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function attention_bias_to_padding at 0x7ff00d7338c8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function attention_bias_to_padding at 0x7ff00d7338c8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function attention_bias_to_padding at 0x7ff00d7338c8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/expert_utils.py:621: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/utils/expert_utils.py:621: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.squared_difference is deprecated. Please use tf.math.squared_difference instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.squared_difference is deprecated. Please use tf.math.squared_difference instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.rsqrt is deprecated. Please use tf.math.rsqrt instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.rsqrt is deprecated. Please use tf.math.rsqrt instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.get_collection is deprecated. Please use tf.compat.v1.get_collection instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.get_collection is deprecated. Please use tf.compat.v1.get_collection instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.GraphKeys is deprecated. Please use tf.compat.v1.GraphKeys instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.GraphKeys is deprecated. Please use tf.compat.v1.GraphKeys instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function layers at 0x7ff053ac9620> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7fefa041df28>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function layers at 0x7ff053ac9620> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7fefa041df28>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function layers at 0x7ff053ac9620> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7fefa041df28>\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/layers/common_layers.py:3083: The name tf.layers.Dense is deprecated. Please use tf.compat.v1.layers.Dense instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/layers/common_layers.py:3083: The name tf.layers.Dense is deprecated. Please use tf.compat.v1.layers.Dense instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.summary.image is deprecated. Please use tf.compat.v1.summary.image instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.summary.image is deprecated. Please use tf.compat.v1.summary.image instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_26088_1024.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_26088_1024.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/models/transformer.py:1226: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor/models/transformer.py:1226: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_26088_1024.bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_26088_1024.bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_26088_1024.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_26088_1024.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_26088_1024.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_26088_1024.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from t2t/train-large/model.ckpt-225000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from t2t/train-large/model.ckpt-225000\n"
     ]
    }
   ],
   "source": [
    "class Model:\n",
    "    def __init__(self, HPARAMS = \"transformer_big\", DATA_DIR = 't2t/data'):\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        \n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        maxlen_decode = 50 + tf.reduce_max(self.X_seq_len)\n",
    "        \n",
    "        x = tf.expand_dims(tf.expand_dims(self.X, -1), -1)\n",
    "        y = tf.expand_dims(tf.expand_dims(self.Y, -1), -1)\n",
    "        \n",
    "        features = {\n",
    "            \"inputs\": x,\n",
    "            \"targets\": y,\n",
    "            \"target_space_id\": tf.constant(1, dtype=tf.int32),\n",
    "        }\n",
    "        self.features = features\n",
    "        \n",
    "        Modes = tf.estimator.ModeKeys\n",
    "        hparams = trainer_lib.create_hparams(HPARAMS, data_dir=DATA_DIR, problem_name=PROBLEM)\n",
    "        translate_model = registry.model('transformer')(hparams, Modes.PREDICT)\n",
    "        self.translate_model = translate_model\n",
    "        logits, _ = translate_model(features)\n",
    "        \n",
    "        with tf.variable_scope(tf.get_variable_scope(), reuse=True):\n",
    "            self.fast_result = translate_model._greedy_infer(features, maxlen_decode)[\"outputs\"]\n",
    "            self.beam_result = translate_model._beam_decode_slow(\n",
    "                features, maxlen_decode, beam_size=5, \n",
    "                top_beams=1, alpha=1.0)[\"outputs\"]\n",
    "        \n",
    "        self.fast_result = tf.identity(self.fast_result, name = 'greedy')\n",
    "        self.beam_result = tf.identity(self.beam_result, name = 'beam')\n",
    "        \n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model()\n",
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, ckpt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "\n",
    "def cleaning(string):\n",
    "    return re.sub(r'[ ]+', ' ', unidecode(string)).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('WHEN ALICIA GARZA wrote “Black lives matter” in a Facebook post nearly seven '\n",
      " 'years ago, the activist from Oakland, California, never imagined that those '\n",
      " 'words would come to define a global movement. The July 13, 2013, acquittal '\n",
      " 'of George Zimmerman for the killing of Trayvon Martin, an unarmed Black '\n",
      " 'teenager, sparked her post.')\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "string = 'WHEN ALICIA GARZA wrote “Black lives matter” in a Facebook post nearly seven years ago, the activist from Oakland, California, never imagined that those words would come to define a global movement. The July 13, 2013, acquittal of George Zimmerman for the killing of Trayvon Martin, an unarmed Black teenager, sparked her post.'\n",
    "pprint(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('KETIKA ALICIA GARZA menulis \"Kehidupan hitam penting\" dalam catatan Facebook hampir tujuh tahun yang lalu, aktivis dari Oakland, California, tidak pernah membayangkan bahawa kata-kata itu akan menentukan pergerakan global. 13 Julai 2013, pembebasan George Zimmerman atas pembunuhan Trayvon Martin, seorang remaja kulit hitam yang tidak bersenjata, mencetuskan jawatannya.<EOS>',\n",
       " 'KETIKA ALICIA GARZA menulis \"Kehidupan hitam penting\" dalam catatan Facebook hampir tujuh tahun yang lalu, aktivis dari Oakland, California, tidak pernah membayangkan bahawa kata-kata itu akan menentukan pergerakan global. 13 Julai 2013, pembebasan George Zimmerman atas pembunuhan Trayvon Martin, seorang remaja kulit hitam yang tidak bersenjata, mencetuskan jawatannya.<EOS><pad><pad><pad>')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded = encoder.encode(cleaning(string)) + [1]\n",
    "f, b = sess.run([model.fast_result, model.beam_result], feed_dict = {model.X: [encoded]})\n",
    "encoder.decode(f[0]), encoder.decode(b[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('KUALA LUMPUR 1 Julai - Datuk Seri Anwar Ibrahim tidak sesuai menjadi calon Perdana Menteri kerana beliau didakwa tidak \"popular\" dalam kalangan orang Melayu, kata Tun Dr Mahathir Mohamad. Bekas Perdana Menteri itu dilaporkan berkata, Presiden PKR memerlukan seseorang seperti dirinya bagi mendapatkan sokongan daripada orang Melayu dan memenangi pilihan raya.<EOS>',\n",
       " 'KUALA LUMPUR 1 Julai - Datuk Seri Anwar Ibrahim tidak sesuai menjadi calon Perdana Menteri kerana beliau didakwa tidak \"popular\" dalam kalangan orang Melayu, kata Tun Dr Mahathir Mohamad. Bekas Perdana Menteri itu dilaporkan berkata, Presiden PKR memerlukan seseorang seperti dirinya bagi mendapatkan sokongan daripada orang Melayu dan memenangi pilihan raya.<EOS><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'KUALA LUMPUR, July 1 - Datuk Seri Anwar Ibrahim is not suitable to as the prime minister candidate as he is allegedly not \"popular\" among the Malays, Tun Dr Mahathir Mohamad claimed. The former prime minister reportedly said the PKR president needs someone like himself in order to acquire support from the Malays and win the election.'\n",
    "encoded = encoder.encode(string) + [1]\n",
    "f, b = sess.run([model.fast_result, model.beam_result], feed_dict = {model.X: [encoded]})\n",
    "encoder.decode(f[0]), encoder.decode(b[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Agama biasanya bermaksud kepercayaan kepada Tuhan, atau kekuatan ghaib dan ghaib seperti Tuhan, serta amalan dan institusi yang berkaitan dengan kepercayaan itu. Agama dan kepercayaan adalah dua perkara yang sangat relevan. Tetapi Agama mempunyai makna yang lebih luas, yang merujuk kepada sistem kepercayaan yang padu, dan kepercayaan ini adalah mengenai aspek ilahi<EOS>',\n",
       " 'Agama biasanya bermaksud kepercayaan kepada Tuhan, atau kekuatan ghaib dan ghaib seperti Tuhan, serta amalan dan institusi yang berkaitan dengan kepercayaan itu. Agama dan kepercayaan adalah dua perkara yang sangat relevan. Tetapi Agama mempunyai makna yang lebih luas, yang merujuk kepada sistem kepercayaan yang padu, dan kepercayaan ini adalah mengenai aspek ilahi<EOS><pad><pad><pad>')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'Religion usually means trust in God, or a supernatural and supernatural power like God, as well as practices and institutions associated with that belief. Religion and belief are two very relevant things. But Religion has a broader meaning, which refers to a system of cohesive belief, and this belief is about the divine aspect'\n",
    "encoded = encoder.encode(string) + [1]\n",
    "f, b = sess.run([model.fast_result, model.beam_result], feed_dict = {model.X: [encoded]})\n",
    "encoder.decode(f[0]), encoder.decode(b[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Articulate (Well-spoken, Expressive) <> Hmmm ya Horacio Calcaterra adalah seorang olahragawan yang saat ini bermain untuk Sporting Cristal yang terletak di Torneo Descentralizado. <> Dia kini bermain untuk Sporting Cristal di Torneo Descentralizado.<EOS>',\n",
       " 'Articulate (Well-spoken, Expressive) <> Hmmm ya Horacio Calcaterra adalah ahli sukan yang kini bermain untuk Sporting Cristal yang terletak di Torneo Descentralizado. <> Dia kini bermain untuk Sporting Cristal di Torneo Descentralizado.<EOS><pad><pad><pad><pad>')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'Articulate (Well-spoken, Expressive) <> Hmmm yes Horacio Calcaterra is a sportsman that currently is playing for Sporting Cristal which is located in Torneo Descentralizado. <> He currently plays for Sporting Cristal in the Torneo Descentralizado.'\n",
    "encoded = encoder.encode(string) + [1]\n",
    "f, b = sess.run([model.fast_result, model.beam_result], feed_dict = {model.X: [encoded]})\n",
    "encoder.decode(f[0]), encoder.decode(b[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('TANGKAK - Tan Sri Muhyiddin Yassin berkata, beliau tidak mahu menyentuh isu politik pada masa ini, sebaliknya menumpukan kepada kebajikan rakyat dan usaha memulihkan ekonomi negara yang terjejas susulan wabak Covid-19, Perdana Menteri menjelaskan perkara itu ketika berucap pada Mesyuarat Kepimpinan Bersama Pemimpin Dewan Undangan Negeri (DUN) Gambir di Dewan Serbaguna Bukit Gambir hari ini.<EOS>',\n",
       " 'TANGKAK - Tan Sri Muhyiddin Yassin berkata, beliau tidak mahu menyentuh isu politik pada masa ini, sebaliknya memberi tumpuan kepada kebajikan rakyat dan usaha memulihkan ekonomi negara yang terjejas susulan wabak Covid-19, Perdana Menteri menjelaskan perkara itu ketika berucap pada Mesyuarat Pemimpin Bersama Pemimpin Dewan Undangan Negeri (DUN) Gambir di Dewan Serbaguna Bukit Gambir, hari ini.<EOS><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = ('TANGKAK - Tan Sri Muhyiddin Yassin said he did not want to touch on '\n",
    " 'political issues at the moment, instead focusing on the welfare of the '\n",
    " \"people and efforts to revitalize the affected country's economy following \"\n",
    " 'the Covid-19 pandemic. The prime minister explained the matter when speaking '\n",
    " 'at a Leadership Meeting with Gambir State Assembly (DUN) leaders at the '\n",
    " 'Bukit Gambir Multipurpose Hall today.')\n",
    "encoded = encoder.encode(string) + [1]\n",
    "f, b = sess.run([model.fast_result, model.beam_result], feed_dict = {model.X: [encoded]})\n",
    "encoder.decode(f[0]), encoder.decode(b[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('saya di sekolah perubatan.<EOS>', 'saya di sekolah perubatan.<EOS><pad>')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = \"i am in medical school.\"\n",
    "string = unidecode(string)\n",
    "encoded = encoder.encode(string) + [1]\n",
    "f, b = sess.run([model.fast_result, model.beam_result], feed_dict = {model.X: [encoded]})\n",
    "encoder.decode(f[0]), encoder.decode(b[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Emmerdale adalah album studio debut, lagu-lagu tidak dikeluarkan di AS <> Lagu-lagu ini tidak dikeluarkan dalam edisi A.S. album tersebut dan sebelumnya tidak tersedia untuk sebarang pelepasan A.S.<EOS>',\n",
       " 'Emmerdale adalah album studio debut, lagu-lagu tidak dirilis di AS <> Lagu-lagu ini tidak dirilis dalam edisi A.S. album tersebut dan sebelumnya tidak tersedia untuk pelepasan A.S.<EOS><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'Emmerdale is the debut studio album,songs were not released in the U.S <> These songs were not released in the U.S. edition of said album and were previously unavailable on any U.S. release.'\n",
    "encoded = encoder.encode(string) + [1]\n",
    "f, b = sess.run([model.fast_result, model.beam_result], feed_dict = {model.X: [encoded]})\n",
    "encoder.decode(f[0]), encoder.decode(b[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Pemadanan kabur pada skala. Dari 3.7 jam hingga 0.2 saat. Cara melakukan pemadanan rentetan pintar dengan cara yang dapat meningkatkan skala ke set data terbesar. Data di dunia nyata tidak kemas. Berurusan dengan set data yang tidak kemas menyakitkan dan terbakar sepanjang masa yang dapat dihabiskan untuk menganalisis data itu sendiri.<EOS>',\n",
       " 'Pemadanan kabur pada skala. Dari 3.7 jam hingga 0.2 saat. Cara melakukan pemadanan rentetan pintar dengan cara yang dapat meningkatkan skala ke set data terbesar. Data di dunia nyata tidak kemas. Berurusan dengan set data yang tidak kemas menyakitkan dan terbakar sepanjang masa yang dapat dihabiskan untuk menganalisis data itu sendiri.<EOS><pad>')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://towardsdatascience.com/fuzzy-matching-at-scale-84f2bfd0c536\n",
    "\n",
    "string = 'Fuzzy matching at scale. From 3.7 hours to 0.2 seconds. How to perform intelligent string matching in a way that can scale to even the biggest data sets. Data in the real world is messy. Dealing with messy data sets is painful and burns through time which could be spent analysing the data itself.'\n",
    "encoded = encoder.encode(string) + [1]\n",
    "f, b = sess.run([model.fast_result, model.beam_result], feed_dict = {model.X: [encoded]})\n",
    "encoder.decode(f[0]), encoder.decode(b[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(77707, 77707)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 128\n",
    "\n",
    "path = 't2t/tmp/test-en'\n",
    "\n",
    "with open(os.path.join(path, 'left.txt')) as fopen:\n",
    "    left = fopen.read().split('\\n')\n",
    "    \n",
    "with open(os.path.join(path, 'right.txt')) as fopen:\n",
    "    right = fopen.read().split('\\n')\n",
    "    \n",
    "len(left), len(right)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[453,\n",
       "  389,\n",
       "  764,\n",
       "  6870,\n",
       "  9233,\n",
       "  90,\n",
       "  1571,\n",
       "  5,\n",
       "  3759,\n",
       "  65,\n",
       "  14878,\n",
       "  4225,\n",
       "  3794,\n",
       "  2,\n",
       "  564,\n",
       "  781,\n",
       "  22434,\n",
       "  1233,\n",
       "  4328,\n",
       "  28,\n",
       "  546,\n",
       "  46,\n",
       "  5,\n",
       "  443,\n",
       "  2344,\n",
       "  114,\n",
       "  241,\n",
       "  133,\n",
       "  2,\n",
       "  11911,\n",
       "  2,\n",
       "  11907,\n",
       "  2,\n",
       "  11911,\n",
       "  261,\n",
       "  14,\n",
       "  613,\n",
       "  20604,\n",
       "  21054,\n",
       "  2038,\n",
       "  14,\n",
       "  1999,\n",
       "  8,\n",
       "  447,\n",
       "  28,\n",
       "  114,\n",
       "  241,\n",
       "  133,\n",
       "  5,\n",
       "  8532,\n",
       "  2,\n",
       "  46,\n",
       "  110,\n",
       "  1411,\n",
       "  2197,\n",
       "  8,\n",
       "  2857,\n",
       "  234,\n",
       "  23554,\n",
       "  11,\n",
       "  6025,\n",
       "  15,\n",
       "  153,\n",
       "  103,\n",
       "  4169,\n",
       "  8,\n",
       "  3308,\n",
       "  2,\n",
       "  125,\n",
       "  30,\n",
       "  49,\n",
       "  1922,\n",
       "  8,\n",
       "  15294,\n",
       "  7456,\n",
       "  94,\n",
       "  901,\n",
       "  9192,\n",
       "  2,\n",
       "  1922,\n",
       "  8,\n",
       "  901,\n",
       "  1006,\n",
       "  7783,\n",
       "  14543,\n",
       "  2,\n",
       "  901,\n",
       "  1770,\n",
       "  35,\n",
       "  10616,\n",
       "  19513,\n",
       "  336,\n",
       "  7814,\n",
       "  13234,\n",
       "  14,\n",
       "  901,\n",
       "  11161,\n",
       "  8655,\n",
       "  94,\n",
       "  1497,\n",
       "  5215,\n",
       "  3141,\n",
       "  8,\n",
       "  7644,\n",
       "  19,\n",
       "  163,\n",
       "  20694,\n",
       "  80,\n",
       "  31,\n",
       "  1927,\n",
       "  55,\n",
       "  658,\n",
       "  16,\n",
       "  2477,\n",
       "  6238,\n",
       "  11365,\n",
       "  54,\n",
       "  22990,\n",
       "  6,\n",
       "  1674,\n",
       "  77,\n",
       "  18744,\n",
       "  6,\n",
       "  11,\n",
       "  6025,\n",
       "  15,\n",
       "  144,\n",
       "  1246,\n",
       "  4182,\n",
       "  301,\n",
       "  1041,\n",
       "  2,\n",
       "  227,\n",
       "  163,\n",
       "  4642,\n",
       "  1405,\n",
       "  3141,\n",
       "  8,\n",
       "  34,\n",
       "  110,\n",
       "  4297,\n",
       "  5,\n",
       "  356,\n",
       "  999,\n",
       "  1672,\n",
       "  1147,\n",
       "  141,\n",
       "  52,\n",
       "  12274,\n",
       "  593,\n",
       "  12310,\n",
       "  2,\n",
       "  8,\n",
       "  1747,\n",
       "  19842,\n",
       "  3141,\n",
       "  2545,\n",
       "  14,\n",
       "  2458,\n",
       "  16,\n",
       "  4055,\n",
       "  14800,\n",
       "  3010,\n",
       "  3919,\n",
       "  3454,\n",
       "  17034,\n",
       "  11050,\n",
       "  35,\n",
       "  232,\n",
       "  2489,\n",
       "  30,\n",
       "  2,\n",
       "  52,\n",
       "  371,\n",
       "  107,\n",
       "  5806,\n",
       "  13902,\n",
       "  6723,\n",
       "  27,\n",
       "  28,\n",
       "  11546,\n",
       "  42,\n",
       "  8245,\n",
       "  14,\n",
       "  12577,\n",
       "  310,\n",
       "  2766,\n",
       "  372,\n",
       "  12,\n",
       "  5436,\n",
       "  6700,\n",
       "  2203,\n",
       "  310,\n",
       "  23965,\n",
       "  6,\n",
       "  2254,\n",
       "  5,\n",
       "  6193,\n",
       "  163,\n",
       "  505,\n",
       "  20055,\n",
       "  14746,\n",
       "  1922,\n",
       "  8,\n",
       "  241,\n",
       "  113]]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = sess.run(model.fast_result, feed_dict = {model.X: [encoder.encode(left[0]) + [1]]}).tolist()\n",
    "results = []\n",
    "for row in p:\n",
    "    results.append([i for i in row if i not in [0, 1]])\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensor2tensor.utils import bleu_hook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6865377"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bleu_hook.compute_bleu(reference_corpus = [encoder.encode(right[0])], \n",
    "                       translation_corpus = results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 608/608 [1:32:22<00:00,  9.12s/it]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "results = []\n",
    "for i in tqdm(range(0, len(left), batch_size)):\n",
    "    index = min(i + batch_size, len(left))\n",
    "    x = left[i: index]\n",
    "    encoded = [encoder.encode(l) + [1] for l in x]\n",
    "    batch_x = pad_sequences(encoded, padding='post')\n",
    "    \n",
    "    p = sess.run(model.fast_result, feed_dict = {model.X: batch_x}).tolist()\n",
    "    result = []\n",
    "    for row in p:\n",
    "        result.append([i for i in row if i not in [0, 1]])\n",
    "    results.extend(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "77707"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.698501"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rights = [encoder.encode(r) for r in right[:len(results)]]\n",
    "bleu_hook.compute_bleu(reference_corpus = rights,\n",
    "                       translation_corpus = results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'transformer-large/model.ckpt'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'transformer-large/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'transformer/body/target_space_embedding/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/target_space_embedding/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/target_space_embedding/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/decoder/layer_0/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_0/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_1/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_1/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_2/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_2/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_3/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_3/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_4/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_4/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_5/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_5/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/target_space_embedding/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'greedy',\n",
       " 'beam']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'greedy' in n.name\n",
    "        or 'beam' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'modality' not in n.name\n",
    "        and 'Assign' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from transformer-large/model.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from transformer-large/model.ckpt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-27-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-27-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Froze 201 variables.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Froze 201 variables.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Converted 201 variables to const ops.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Converted 201 variables to const ops.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11006 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('transformer-large', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "g = load_graph('transformer-large/frozen_model.pb')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "greedy = g.get_tensor_by_name('import/greedy:0')\n",
    "beam = g.get_tensor_by_name('import/beam:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = 'KUALA LUMPUR, July 1 - Datuk Seri Anwar Ibrahim is not suitable to as the prime minister candidate as he is allegedly not \"popular\" among the Malays, Tun Dr Mahathir Mohamad claimed. The former prime minister reportedly said the PKR president needs someone like himself in order to acquire support from the Malays and win the election.'\n",
    "encoded = encoder.encode(string) + [1]\n",
    "g, b = test_sess.run([greedy, beam], feed_dict = {x:[encoded]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('KUALA LUMPUR 1 Julai - Datuk Seri Anwar Ibrahim tidak sesuai menjadi calon Perdana Menteri kerana beliau didakwa tidak \"popular\" dalam kalangan orang Melayu, kata Tun Dr Mahathir Mohamad. Bekas Perdana Menteri itu dilaporkan berkata, Presiden PKR memerlukan seseorang seperti dirinya bagi mendapatkan sokongan daripada orang Melayu dan memenangi pilihan raya.<EOS>',\n",
       " 'KUALA LUMPUR 1 Julai - Datuk Seri Anwar Ibrahim tidak sesuai menjadi calon Perdana Menteri kerana beliau didakwa tidak \"popular\" dalam kalangan orang Melayu, kata Tun Dr Mahathir Mohamad. Bekas Perdana Menteri itu dilaporkan berkata, Presiden PKR memerlukan seseorang seperti dirinya bagi mendapatkan sokongan daripada orang Melayu dan memenangi pilihan raya.<EOS><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder.decode(g[0]), encoder.decode(b[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
